{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/VyacheslavGusev/Machine_Lerning/blob/main/PySpark_log-reg_%26_TreeClass.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q5XdW4eUdiwJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b3dd625-f605-47a8-e6f8-4f175357f2b4"
      },
      "source": [
        "!apt-get update"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r0% [Working]\r            \rGet:1 https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/ InRelease [3,622 B]\n",
            "\r0% [Waiting for headers] [Connecting to security.ubuntu.com (185.125.190.39)] [\r0% [Waiting for headers] [Connecting to security.ubuntu.com (185.125.190.39)] [\r                                                                               \rHit:2 http://archive.ubuntu.com/ubuntu focal InRelease\n",
            "\r0% [Waiting for headers] [Waiting for headers] [Connecting to ppa.launchpad.net\r                                                                               \rHit:3 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64  InRelease\n",
            "\r0% [Waiting for headers] [Waiting for headers] [Connecting to ppa.launchpad.net\r                                                                               \rGet:4 http://archive.ubuntu.com/ubuntu focal-updates InRelease [114 kB]\n",
            "Get:5 http://security.ubuntu.com/ubuntu focal-security InRelease [114 kB]\n",
            "Get:6 http://archive.ubuntu.com/ubuntu focal-backports InRelease [108 kB]\n",
            "Hit:7 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu focal InRelease\n",
            "Hit:8 http://ppa.launchpad.net/cran/libgit2/ubuntu focal InRelease\n",
            "Get:9 http://archive.ubuntu.com/ubuntu focal-updates/main amd64 Packages [3,175 kB]\n",
            "Hit:10 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu focal InRelease\n",
            "Get:11 http://archive.ubuntu.com/ubuntu focal-updates/universe amd64 Packages [1,344 kB]\n",
            "Hit:12 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu focal InRelease\n",
            "Hit:13 http://ppa.launchpad.net/ubuntugis/ppa/ubuntu focal InRelease\n",
            "Get:14 http://security.ubuntu.com/ubuntu focal-security/restricted amd64 Packages [2,240 kB]\n",
            "Get:15 http://security.ubuntu.com/ubuntu focal-security/main amd64 Packages [2,699 kB]\n",
            "Get:16 http://security.ubuntu.com/ubuntu focal-security/universe amd64 Packages [1,049 kB]\n",
            "Fetched 10.8 MB in 2s (4,416 kB/s)\n",
            "Reading package lists... Done\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1HyDmS5WduU3"
      },
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9yRaYSD6dux4"
      },
      "source": [
        "!wget -q https://downloads.apache.org/spark/spark-3.4.0/spark-3.4.0-bin-hadoop3.tgz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z2MaC-k0dx8l",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "60e9a762-080b-4959-8ee6-d90009397731"
      },
      "source": [
        "!tar -xvf spark-3.4.0-bin-hadoop3.tgz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "spark-3.4.0-bin-hadoop3/\n",
            "spark-3.4.0-bin-hadoop3/data/\n",
            "spark-3.4.0-bin-hadoop3/data/streaming/\n",
            "spark-3.4.0-bin-hadoop3/data/streaming/AFINN-111.txt\n",
            "spark-3.4.0-bin-hadoop3/data/graphx/\n",
            "spark-3.4.0-bin-hadoop3/data/graphx/followers.txt\n",
            "spark-3.4.0-bin-hadoop3/data/graphx/users.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/als/\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/als/test.data\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/als/sample_movielens_ratings.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/gmm_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_libsvm_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_svm_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_fpgrowth.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/kmeans_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/pic_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/streaming_kmeans_data_test.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_linear_regression_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/ridge-data/\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/ridge-data/lpsa.data\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_binary_classification_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/pagerank_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_isotonic_regression_libsvm_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_kmeans_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_lda_libsvm_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/license.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/multi-channel/\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/multi-channel/chr30.4.184.jpg\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/multi-channel/BGRA_alpha_60.png\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/multi-channel/grayscale.jpg\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/multi-channel/BGRA.png\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/license.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/kittens/\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/kittens/DP802813.jpg\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/kittens/54893.jpg\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/kittens/not-image.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/kittens/DP153539.jpg\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/images/origin/kittens/29.5.a_b_EGDP022204.jpg\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_lda_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_movielens_data.txt\n",
            "spark-3.4.0-bin-hadoop3/data/mllib/sample_multiclass_classification_data.txt\n",
            "spark-3.4.0-bin-hadoop3/LICENSE\n",
            "spark-3.4.0-bin-hadoop3/jars/\n",
            "spark-3.4.0-bin-hadoop3/jars/zookeeper-jute-3.6.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/derby-10.14.2.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-shims-0.23-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/oro-2.0.8.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jline-2.14.6.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-buffer-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/libthrift-0.12.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/json-1.8.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-events-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/avro-ipc-1.11.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jodd-core-3.5.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jackson-dataformat-yaml-2.14.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-node-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-graphx_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-compiler-3.1.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/javolution-5.5.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jersey-hk2-2.36.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-transport-native-epoll-4.1.87.Final-linux-x86_64.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/opencsv-2.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/orc-core-1.8.3-shaded-protobuf.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-transport-native-epoll-4.1.87.Final-linux-aarch_64.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/zjsonpatch-0.3.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-collections-3.2.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/json4s-scalap_2.12-3.7.0-M11.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jackson-core-asl-1.9.13.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jersey-container-servlet-2.36.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jackson-annotations-2.14.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/JLargeArrays-1.5.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/antlr-runtime-3.5.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/arpack-3.0.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/annotations-17.0.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-mllib-local_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jul-to-slf4j-2.0.6.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/guava-14.0.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/mesos-1.4.3-shaded-protobuf.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/orc-shims-1.8.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-storage-api-2.8.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jackson-datatype-jsr310-2.14.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/xbean-asm9-shaded-4.22.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/breeze_2.12-2.1.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/zookeeper-3.6.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/parquet-column-1.12.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-mllib_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spire-macros_2.12-0.17.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-compress-1.22.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/ST4-4.0.4.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hadoop-client-api-3.3.4.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/arrow-vector-11.0.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/curator-framework-2.13.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/antlr4-runtime-4.9.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/log4j-slf4j2-impl-2.19.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jersey-client-2.36.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jackson-databind-2.14.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/scala-library-2.12.17.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-networking-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-llap-common-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/scala-compiler-2.12.17.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/bonecp-0.8.0.RELEASE.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-mesos_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/snappy-java-1.1.9.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-codec-http2-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-common-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spire-platform_2.12-0.17.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/parquet-jackson-1.12.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/lz4-java-1.8.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/curator-client-2.13.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jdo-api-3.0.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-serde-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/httpcore-4.4.16.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/transaction-api-1.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-handler-proxy-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/univocity-parsers-2.9.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-flowcontrol-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-lang-2.6.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-collections4-4.4.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-sketch_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.87.Final-osx-x86_64.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/metrics-jvm-4.2.15.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jsr305-3.0.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-network-shuffle_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/py4j-0.10.9.7.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jta-1.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/HikariCP-2.5.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-catalyst_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/scala-reflect-2.12.17.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-pool-1.5.4.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/JTransforms-3.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-math3-3.6.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/flatbuffers-java-1.12.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/rocksdbjni-7.9.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/okio-1.15.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jersey-server-2.36.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/json4s-core_2.12-3.7.0-M11.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/datanucleus-core-4.1.17.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/threeten-extra-1.7.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spire_2.12-0.17.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jackson-core-2.14.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hadoop-client-runtime-3.3.4.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/osgi-resource-locator-1.0.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/cats-kernel_2.12-2.1.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/paranamer-2.8.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/pickle-1.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-all-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-streaming_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spire-util_2.12-0.17.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/janino-3.1.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/curator-recipes-2.13.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jersey-common-2.36.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-common-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hk2-locator-2.6.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jakarta.annotation-api-1.3.5.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jpam-1.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.87.Final-osx-aarch_64.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-client-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/orc-mapreduce-1.8.3-shaded-protobuf.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/metrics-core-4.2.15.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/datanucleus-rdbms-4.1.19.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-repl_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jakarta.inject-2.6.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/snakeyaml-1.33.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/minlog-1.3.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/httpclient-4.5.14.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/parquet-format-structures-1.12.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hk2-utils-2.6.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-cli-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-codec-http-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-httpclient-okhttp-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jackson-module-scala_2.12-2.14.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/audience-annotations-0.5.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/arrow-memory-netty-11.0.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-common-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-transport-classes-epoll-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-certificates-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/parquet-hadoop-1.12.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-core-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-service-rpc-3.1.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-kubernetes_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-admissionregistration-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-codec-1.15.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/objenesis-3.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jackson-mapper-asl-1.9.13.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-crypto-1.1.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-network-common_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-shims-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/aircompressor-0.21.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/xz-1.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-policy-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-rbac-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/scala-parser-combinators_2.12-2.1.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/arrow-memory-core-11.0.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-apiextensions-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jersey-container-servlet-core-2.36.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/protobuf-java-2.5.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-logging-1.1.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-cli-1.5.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-unsafe_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/log4j-1.2-api-2.19.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jakarta.validation-api-2.0.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-kvstore_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/blas-3.0.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/stax-api-1.0.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-autoscaling-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-dbcp-1.4.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/RoaringBitmap-0.9.38.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/slf4j-api-2.0.6.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/parquet-encoding-1.12.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-codec-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-batch-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/activation-1.1.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/aopalliance-repackaged-2.6.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/metrics-jmx-4.2.15.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/ivy-2.5.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-sql_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jakarta.servlet-api-4.0.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-extensions-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-metastore-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/tink-1.7.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/gson-2.2.4.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/zstd-jni-1.5.2-5.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-shims-scheduler-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-core_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-transport-native-unix-common-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hadoop-shaded-guava-1.1.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/compress-lzf-1.1.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/arrow-format-11.0.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/logging-interceptor-3.12.12.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-beeline-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-launcher_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jakarta.ws.rs-api-2.1.6.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-apps-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/json4s-jackson_2.12-3.7.0-M11.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-tags_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/okhttp-3.12.12.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jcl-over-slf4j-2.0.6.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-metrics-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/istack-commons-runtime-3.0.8.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-jdbc-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/log4j-api-2.19.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/javax.jdo-3.2.0-m3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-codec-socks-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/leveldbjni-all-1.8.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/stream-2.9.6.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/avro-mapred-1.11.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/avro-1.11.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/libfb303-0.9.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/super-csv-2.2.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-storageclass-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-transport-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-resolver-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jaxb-runtime-2.3.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/scala-xml_2.12-2.1.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-coordination-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-scheduling-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/chill-java-0.10.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-client-api-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-handler-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/breeze-macros_2.12-2.1.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/algebra_2.12-2.0.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-lang3-3.12.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/scala-collection-compat_2.12-2.7.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-io-2.11.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-yarn_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/jakarta.xml.bind-api-2.3.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kryo-shaded-4.0.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hk2-api-2.6.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-hive_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-shims-common-2.3.9.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hive-exec-2.3.9-core.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-gatewayapi-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/spark-hive-thriftserver_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/metrics-graphite-4.2.15.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/chill_2.12-0.10.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/shims-0.9.38.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/datanucleus-api-jdo-4.2.4.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/hadoop-yarn-server-web-proxy-3.3.4.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/netty-transport-classes-kqueue-4.1.87.Final.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/metrics-json-4.2.15.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/json4s-ast_2.12-3.7.0-M11.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/kubernetes-model-discovery-6.4.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/javassist-3.25.0-GA.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/joda-time-2.12.2.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/log4j-core-2.19.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/arpack_combined_all-0.1.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/commons-text-1.10.0.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/parquet-common-1.12.3.jar\n",
            "spark-3.4.0-bin-hadoop3/jars/lapack-3.0.3.jar\n",
            "spark-3.4.0-bin-hadoop3/R/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/sparkr.zip\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/NAMESPACE\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/worker/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/worker/worker.R\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/worker/daemon.R\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/profile/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/profile/general.R\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/profile/shell.R\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/tests/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/tests/testthat/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/tests/testthat/test_basic.R\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/R/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/R/SparkR.rdx\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/R/SparkR.rdb\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/R/SparkR\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/Meta/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/Meta/package.rds\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/Meta/features.rds\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/Meta/vignette.rds\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/Meta/nsInfo.rds\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/Meta/Rd.rds\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/Meta/hsearch.rds\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/Meta/links.rds\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/html/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/html/R.css\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/html/00Index.html\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/doc/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.Rmd\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.R\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.html\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/doc/index.html\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/INDEX\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/DESCRIPTION\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/help/\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/help/SparkR.rdx\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/help/aliases.rds\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/help/SparkR.rdb\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/help/paths.rds\n",
            "spark-3.4.0-bin-hadoop3/R/lib/SparkR/help/AnIndex\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/dockerfiles/\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/dockerfiles/spark/\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/dockerfiles/spark/decom.sh\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/dockerfiles/spark/bindings/\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/dockerfiles/spark/bindings/R/\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/dockerfiles/spark/bindings/R/Dockerfile\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/dockerfiles/spark/bindings/python/\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/dockerfiles/spark/bindings/python/Dockerfile\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/dockerfiles/spark/entrypoint.sh\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/dockerfiles/spark/Dockerfile\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/tests/\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/tests/py_container_checks.py\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/tests/decommissioning.py\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/tests/autoscale.py\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/tests/pyfiles.py\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/tests/worker_memory_check.py\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/tests/python_executable_check.py\n",
            "spark-3.4.0-bin-hadoop3/kubernetes/tests/decommissioning_cleanup.py\n",
            "spark-3.4.0-bin-hadoop3/examples/\n",
            "spark-3.4.0-bin-hadoop3/examples/jars/\n",
            "spark-3.4.0-bin-hadoop3/examples/jars/scopt_2.12-3.7.1.jar\n",
            "spark-3.4.0-bin-hadoop3/examples/jars/spark-examples_2.12-3.4.0.jar\n",
            "spark-3.4.0-bin-hadoop3/examples/src/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scripts/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scripts/getGpusResources.sh\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/streaming/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/streaming/structured_network_wordcount.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/RSparkSQLExample.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/dataframe.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/data-manipulation.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/decisionTree.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/logit.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/kstest.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/gaussianMixture.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/fmRegressor.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/naiveBayes.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/ml.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/lda.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/kmeans.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/isoreg.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/lm_with_elastic_net.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/als.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/fpm.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/glm.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/bisectingKmeans.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/svmLinear.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/prefixSpan.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/powerIterationClustering.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/fmClassifier.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/randomForest.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/gbt.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/survreg.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/r/ml/mlp.R\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/GroupByTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkPageRank.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedScalar.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredNetworkWordCountWindowed.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredSessionization.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredKerberizedKafkaWordCount.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredComplexSessionization.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredNetworkWordCount.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredKafkaWordCount.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedUntypedAggregation.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedTypedAggregation.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/RDDRelation.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/SimpleTypedAggregator.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/jdbc/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/jdbc/ExampleJdbcConnectionProvider.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/hive/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SkewedGroupByTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkALS.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LocalPi.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/clickstream/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/clickstream/PageViewStream.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/clickstream/PageViewGenerator.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/DirectKafkaWordCount.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/StreamingExamples.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/QueueStream.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/DirectKerberizedKafkaWordCount.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/HdfsWordCount.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/RawNetworkGrep.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ExceptionHandlingTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkRemoteFileTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/DFSReadWriteTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkTC.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/BroadcastTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkKMeans.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/HdfsTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LocalLR.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LocalKMeans.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/LiveJournalPageRank.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/ConnectedComponentsExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/ComprehensiveExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/SSSPExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/SynthBenchmark.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/PageRankExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/TriangleCountingExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/Analytics.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkHdfsLR.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/extensions/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/extensions/SparkSessionExtensionsTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/extensions/SessionExtensionsWithoutLoader.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/extensions/AgeExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/extensions/SessionExtensionsWithLoader.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LocalFileLR.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkLR.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/MultiBroadcastTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LogQuery.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SimpleSkewedGroupByTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LocalALS.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/pythonconverters/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/pythonconverters/AvroConverters.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ChiSquareTestExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/AFTSurvivalRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionWithElasticNetExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/RobustScalerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/CountVectorizerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestClassifierExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/VectorSizeHintExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/MaxAbsScalerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/BisectingKMeansExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaCrossValidationExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/GeneralizedLinearRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/IndexToStringExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/Word2VecExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/FMRegressorExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/UnivariateFeatureSelectorExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/BucketedRandomProjectionLSHExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/VarianceThresholdSelectorExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LinearRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/IsotonicRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ImputerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/PrefixSpanExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/PipelineExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ChiSqSelectorExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaTrainValidationSplitExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/FPGrowthExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/PowerIterationClusteringExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/OneVsRestExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LDAExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LinearSVCExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/FMClassifierExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/CorrelationExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeClassifierExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LinearRegressionWithElasticNetExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionSummaryExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/KMeansExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/QuantileDiscretizerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/MulticlassLogisticRegressionWithElasticNetExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeRegressorExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/MinHashLSHExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ElementwiseProductExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/MultilayerPerceptronClassifierExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeClassificationExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/SQLTransformerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/InteractionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/NaiveBayesExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/FeatureHasherExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/TfIdfExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestRegressorExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/GBTExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/SummarizerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/EstimatorTransformerParamExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/GaussianMixtureExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/MiniReadWriteTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/AccumulatorMetricsTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkPi.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/CosineSimilarity.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/LatentDirichletAllocationExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRunner.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeansExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingLogisticRegression.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/DenseKMeans.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/LogisticRegressionWithLBFGSExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/TallSkinnyPCA.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/TallSkinnySVD.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/LBFGSExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/BisectingKMeansExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StratifiedSamplingExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/MultiLabelMetricsExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SampledRDDs.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestClassificationExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SimpleFPGrowth.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/Word2VecExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/CorrelationsExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/PMMLModelExportExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/IsotonicRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/RandomRDDGeneration.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SVDExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/PrefixSpanExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassification.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/FPGrowthExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingLinearRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/PowerIterationClusteringExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/MulticlassMetricsExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/RecommendationExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SummaryStatisticsExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/MovieLensALS.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/KMeansExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SparseNaiveBayes.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/MultivariateSummarizer.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestRegressionExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SVMWithSGDExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/AssociationRulesExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingTestExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/KernelDensityEstimationExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingClassificationExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/HypothesisTestingExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeClassificationExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/NaiveBayesExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostedTreesRunner.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/Correlations.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/HypothesisTestingKolmogorovSmirnovTestExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/GaussianMixtureExample.scala\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/streaming/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/streaming/structured_sessionization.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/streaming/__init__,py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/streaming/structured_network_wordcount.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/streaming/structured_network_wordcount_session_window.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/datasource.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/basic.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/arrow.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sql/hive.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/kmeans.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/wordcount.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/streaming/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/streaming/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/streaming/sql_network_wordcount.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/streaming/stateful_network_wordcount.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/streaming/network_wordcount.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/streaming/queue_stream.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/streaming/network_wordjoinsentiments.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/streaming/hdfs_wordcount.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/streaming/recoverable_network_wordcount.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/avro_inputformat.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/logistic_regression.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/status_api_demo.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/pagerank.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/parquet_inputformat.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/transitive_closure.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/als.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/sort.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/dct_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/normalizer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/random_forest_classifier_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/standard_scaler_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/estimator_transformer_param_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/binarizer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/dataframe_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/aft_survival_regression.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/kmeans_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/imputer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/cross_validator.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/bucketed_random_projection_lsh_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/univariate_feature_selector_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/rformula_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/onehot_encoder_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/chi_square_test_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/vector_size_hint_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/linear_regression_with_elastic_net.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/decision_tree_classification_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/__init__,py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/feature_hasher_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/count_vectorizer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/tokenizer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/word2vec_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/bucketizer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/stopwords_remover_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/polynomial_expansion_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/elementwise_product_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/pca_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/bisecting_k_means_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/robust_scaler_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/vector_assembler_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/quantile_discretizer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/tf_idf_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/fm_classifier_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/linearsvc.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/min_max_scaler_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/correlation_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/generalized_linear_regression_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/sql_transformer.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/index_to_string_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/vector_indexer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/multilayer_perceptron_classification.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/summarizer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/vector_slicer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/logistic_regression_summary_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/pipeline_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/n_gram_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/naive_bayes_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/logistic_regression_with_elastic_net.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/fpgrowth_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/variance_threshold_selector_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/als_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/fm_regressor_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/train_validation_split.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/decision_tree_regression_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/random_forest_regressor_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/prefixspan_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/min_hash_lsh_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/gaussian_mixture_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/string_indexer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/isotonic_regression_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/interaction_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/power_iteration_clustering_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/chisq_selector_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/lda_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/max_abs_scaler_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/ml/one_vs_rest_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/pi.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/normalizer_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/kmeans.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/standard_scaler_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/word2vec.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/gradient_boosting_classification_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/random_forest_regression_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/regression_metrics_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/decision_tree_classification_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/logistic_regression.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/linear_regression_with_sgd_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/hypothesis_testing_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/multi_class_metrics_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/k_means_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/binary_classification_metrics_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/word2vec_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/recommendation_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/ranking_metrics_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/summary_statistics_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/elementwise_product_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/bisecting_k_means_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/gaussian_mixture_model.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/multi_label_metrics_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/sampled_rdds.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/tf_idf_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/streaming_k_means_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/stratified_sampling_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/streaming_linear_regression_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/kernel_density_estimation_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/correlations.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/random_rdd_generation.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/naive_bayes_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/fpgrowth_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/decision_tree_regression_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/random_forest_classification_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/svm_with_sgd_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/gaussian_mixture_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/isotonic_regression_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/svd_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/power_iteration_clustering_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/pca_rowmatrix_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/correlations_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/python/mllib/gradient_boosting_regression_example.py\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedUntypedAggregation.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredNetworkWordCount.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredNetworkWordCountWindowed.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredComplexSessionization.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredKerberizedKafkaWordCount.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredSessionization.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredKafkaWordCount.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedTypedAggregation.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQLExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/hive/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/hive/JavaSparkHiveExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaStatusTrackerDemo.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaDirectKafkaWordCount.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaQueueStream.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaDirectKerberizedKafkaWordCount.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaStatefulNetworkWordCount.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecord.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaSparkPi.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaWordCount.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaTC.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaLogQuery.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaPageRank.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaHdfsLR.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLinearSVCExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaVarianceThresholdSelectorExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaMinHashLSHExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaUnivariateFeatureSelectorExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLabeledDocument.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionSummaryExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLinearRegressionWithElasticNetExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestRegressorExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaPowerIterationClusteringExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaSummarizerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaQuantileDiscretizerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaMulticlassLogisticRegressionWithElasticNetExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSizeHintExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestClassifierExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaFMRegressorExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaRobustScalerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionWithElasticNetExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaInteractionExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeRegressorExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSquareTestExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaFMClassifierExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaImputerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketedRandomProjectionLSHExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaCrossValidationExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaGeneralizedLinearRegressionExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaCorrelationExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaMultilayerPerceptronClassifierExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaIsotonicRegressionExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaFeatureHasherExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaEstimatorTransformerParamExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaDocument.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaTrainValidationSplitExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaPrefixSpanExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaFPGrowthExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaGaussianMixtureExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaLBFGSExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaKMeansExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaStratifiedSamplingExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaMulticlassClassificationMetricsExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaAssociationRulesExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaSVDExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaRecommendationExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaHypothesisTestingExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaDecisionTreeClassificationExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaMultiLabelClassificationMetricsExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaPowerIterationClusteringExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaKernelDensityEstimationExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaNaiveBayesExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaSVMWithSGDExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaBisectingKMeansExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaLatentDirichletAllocationExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaLogisticRegressionWithLBFGSExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaALS.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaStreamingTestExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaSimpleFPGrowth.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaPCAExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingClassificationExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestRegressionExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaHypothesisTestingKolmogorovSmirnovTestExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaChiSqSelectorExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaDecisionTreeRegressionExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaSummaryStatisticsExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaRankingMetricsExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaElementwiseProductExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaCorrelationsExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaIsotonicRegressionExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaPrefixSpanExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingRegressionExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaGaussianMixtureExample.java\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/users.parquet\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/META-INF/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/META-INF/services/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProvider\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/user.avsc\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/employees.json\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/dir1/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/dir1/file1.parquet\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/dir1/file3.json\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/dir1/dir2/\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/dir1/dir2/file2.parquet\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/users.avro\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/kv1.txt\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/people.json\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/people.txt\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/users.orc\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/full_user.avsc\n",
            "spark-3.4.0-bin-hadoop3/examples/src/main/resources/people.csv\n",
            "spark-3.4.0-bin-hadoop3/conf/\n",
            "spark-3.4.0-bin-hadoop3/conf/metrics.properties.template\n",
            "spark-3.4.0-bin-hadoop3/conf/workers.template\n",
            "spark-3.4.0-bin-hadoop3/conf/spark-defaults.conf.template\n",
            "spark-3.4.0-bin-hadoop3/conf/log4j2.properties.template\n",
            "spark-3.4.0-bin-hadoop3/conf/spark-env.sh.template\n",
            "spark-3.4.0-bin-hadoop3/conf/fairscheduler.xml.template\n",
            "spark-3.4.0-bin-hadoop3/NOTICE\n",
            "spark-3.4.0-bin-hadoop3/yarn/\n",
            "spark-3.4.0-bin-hadoop3/yarn/spark-3.4.0-yarn-shuffle.jar\n",
            "spark-3.4.0-bin-hadoop3/python/\n",
            "spark-3.4.0-bin-hadoop3/python/.gitignore\n",
            "spark-3.4.0-bin-hadoop3/python/setup.cfg\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/install.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/dataframe.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/plan.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/_typing.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/relations_pb2.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/catalog_pb2.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/common_pb2.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/base_pb2.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/expressions_pb2.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/commands_pb2.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/base_pb2.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/example_plugins_pb2.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/commands_pb2.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/expressions_pb2.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/types_pb2.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/relations_pb2.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/common_pb2.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/example_plugins_pb2.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/catalog_pb2.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/types_pb2.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/proto/base_pb2_grpc.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/window.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/readwriter.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/column.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/session.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/udf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/types.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/conf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/group.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/catalog.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/client.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/expressions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/connect/conversion.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/dataframe.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/sql_formatter.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/observation.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/streaming/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/streaming/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/streaming/listener.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/streaming/readwriter.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/streaming/query.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/streaming/state.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/avro/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/avro/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/avro/functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_connect_function.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_group.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_catalog.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_arrow.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_connect_plan.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_conf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_udf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_dataframe.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_map.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_arrow_map.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_udf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_connect_column.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_types.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_datasources.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_column.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_connect_basic.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_client.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_errors.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_serde.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_readwriter.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_conf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_errors.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_session.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_group.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/streaming/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/streaming/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/streaming/test_streaming_listener.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/streaming/test_streaming.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_arrow.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_dataframe.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_catalog.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_arrow_map.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_column.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_types.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_udf_profiler.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_datasources.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_map.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf_window.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_readwriter.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_pandas_sqlmetrics.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_serde.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/typing/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/typing/test_dataframe.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/typing/test_column.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/typing/test_udf.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/typing/test_functions.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/typing/test_readwriter.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/typing/test_session.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_arrow_python_udf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_udf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/tests/test_context.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/context.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/window.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/readwriter.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/column.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/_typing.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/session.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/protobuf/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/protobuf/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/protobuf/functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/serializers.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/map_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/typehints.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/_typing/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/_typing/__init__.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/_typing/protocols/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/_typing/protocols/__init__.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/_typing/protocols/series.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/_typing/protocols/frame.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/functions.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/types.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/group_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/pandas/conversion.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/udf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/types.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/conf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/group.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/sql/catalog.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/serializers.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/testing/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/testing/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/testing/connectutils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/testing/mllibutils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/testing/pandasutils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/testing/mlutils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/testing/utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/testing/streamingutils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/testing/sqlutils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/dstream.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/kinesis.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/tests/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/tests/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/tests/test_listener.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/tests/test_context.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/tests/test_kinesis.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/tests/test_dstream.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/listener.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/context.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/streaming/util.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/resource/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/resource/profile.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/resource/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/resource/tests/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/resource/tests/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/resource/tests/test_resources.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/resource/requests.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/resource/information.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/_globals.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/statcounter.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/cloudpickle/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/cloudpickle/compat.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/cloudpickle/cloudpickle.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/cloudpickle/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/cloudpickle/cloudpickle_fast.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_statcounter.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_conf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_taskcontext.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_profiler.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_rddbarrier.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_util.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_shuffle.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_rdd.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_broadcast.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_serializers.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_memory_profiler.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_worker.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_readwrite.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_pin_thread.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_appsubmit.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_install_spark.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/typing/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/typing/test_rdd.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/typing/test_core.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/typing/test_context.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/typing/test_resultiterable.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_context.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_rddsampler.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_join.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_stage_sched.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/tests/test_daemon.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/taskcontext.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/resultiterable.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/rdd.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/context.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/traceback_utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/status.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/daemon.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/files.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/version.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/__pycache__/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/__pycache__/install.cpython-38.pyc\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/shuffle.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/_typing.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/py.typed\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/broadcast.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/find_spark_home.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/resample.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/_typing.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/sql_formatter.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/typedef/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/typedef/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/typedef/typehints.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/plot/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/plot/core.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/plot/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/plot/matplotlib.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/plot/plotly.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/series.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/supported_api_gen.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_indexing.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_config.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_frame_spark.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_window.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_series.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_groupby_slow.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/plot/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/plot/test_frame_plot.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/plot/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/plot/test_series_plot.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_extension.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_scalars.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_dataframe_conversion.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_dataframe.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_series_string.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_sql.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_stats.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_dataframe_spark_io.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_repr.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_series_conversion.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_ops_on_diff_frames_slow.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_date_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_null_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_base.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/testing_utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_generic_functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_series_datetime.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_groupby.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_dataframe_slow.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_namespace.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_default_index.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_reshape.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_ewm.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_spark_functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_typedef.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_csv.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_expanding.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/indexes/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/indexes/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_category.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_base.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_timedelta.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_datetime.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_categorical.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_internal.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_ops_on_diff_frames.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_rolling.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_indexops_spark.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_numpy_compat.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/tests/test_resample.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/exceptions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/resample.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/series.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/scalars.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/window.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/frame.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/common.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/indexes.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/groupby.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/missing/general_functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/window.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/config.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/timedelta_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/null_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/complex_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/num_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/categorical_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/udt_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/binary_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/base.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/boolean_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/string_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/datetime_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/data_type_ops/date_ops.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/frame.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/categorical.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/strings.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/numpy_compat.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/datetimes.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/sql_processor.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/usage_logging/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/usage_logging/usage_logger.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/usage_logging/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/correlation.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/accessors.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/extensions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/indexes/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/indexes/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/indexes/multi.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/indexes/timedelta.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/indexes/datetimes.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/indexes/numeric.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/indexes/base.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/indexes/category.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/internal.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/base.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/generic.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/groupby.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/namespace.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/spark/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/spark/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/spark/utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/spark/accessors.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/spark/functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/indexing.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/pandas/mlflow.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/join.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/python/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/python/pyspark/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/python/pyspark/shell.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/storagelevel.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/tests/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/tests/test_errors.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/tests/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/exceptions/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/exceptions/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/exceptions/captured.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/exceptions/connect.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/exceptions/base.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/errors/error_classes.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/profiler.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/worker.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/conf.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/pipeline.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tuning.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/clustering.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/param/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/param/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/param/shared.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/param/_shared_params_code_gen.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/stat.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/regression.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_tuning.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_persistence.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_util.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_stat.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_algorithms.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_image.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_model_cache.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_evaluation.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_param.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_feature.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_pipeline.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_training_summary.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/typing/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/typing/test_clustering.yaml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/typing/test_regression.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/typing/test_readable.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/typing/test_classification.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/typing/test_feature.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/typing/test_param.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/typing/test_evaluation.yml\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_base.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_linalg.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tests/test_wrapper.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/recommendation.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/feature.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/linalg/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/linalg/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/common.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/_typing.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/wrapper.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/tree.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/image.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/model_cache.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/evaluation.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/torch/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/torch/distributor.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/torch/torch_run_process_wrapper.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/torch/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/torch/tests/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/torch/tests/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/torch/tests/test_distributor.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/torch/tests/test_log_communication.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/torch/log_communication.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/classification.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/functions.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/base.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/util.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/ml/fpm.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/instrumentation_utils.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/util.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/java_gateway.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/accumulators.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/clustering.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/stat/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/stat/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/stat/test.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/stat/KernelDensity.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/stat/_statistics.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/stat/distribution.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/regression.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/tests/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/tests/test_util.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/tests/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/tests/test_stat.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/tests/test_algorithms.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/tests/test_feature.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/tests/test_streaming_algorithms.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/tests/test_linalg.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/recommendation.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/feature.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/linalg/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/linalg/__init__.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/linalg/distributed.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/common.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/_typing.pyi\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/tree.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/evaluation.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/classification.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/util.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/fpm.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/mllib/random.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/shell.py\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark/rddsampler.py\n",
            "spark-3.4.0-bin-hadoop3/python/.coveragerc\n",
            "spark-3.4.0-bin-hadoop3/python/MANIFEST.in\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark.egg-info/\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark.egg-info/PKG-INFO\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark.egg-info/SOURCES.txt\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark.egg-info/top_level.txt\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark.egg-info/requires.txt\n",
            "spark-3.4.0-bin-hadoop3/python/pyspark.egg-info/dependency_links.txt\n",
            "spark-3.4.0-bin-hadoop3/python/run-tests.py\n",
            "spark-3.4.0-bin-hadoop3/python/run-tests\n",
            "spark-3.4.0-bin-hadoop3/python/lib/\n",
            "spark-3.4.0-bin-hadoop3/python/lib/py4j-0.10.9.7-src.zip\n",
            "spark-3.4.0-bin-hadoop3/python/lib/PY4J_LICENSE.txt\n",
            "spark-3.4.0-bin-hadoop3/python/lib/pyspark.zip\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/people_array_utf16le.json\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/streaming/\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/streaming/text-test.txt\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/people1.json\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/people_array.json\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/people.json\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/text-test.txt\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/orc_partitioned/\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/orc_partitioned/b=1/\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/orc_partitioned/b=1/c=1/\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/orc_partitioned/b=1/c=1/.part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc.crc\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/orc_partitioned/b=1/c=1/part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/orc_partitioned/b=0/\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/orc_partitioned/b=0/c=0/\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/orc_partitioned/b=0/c=0/.part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc.crc\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/orc_partitioned/b=0/c=0/part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/orc_partitioned/_SUCCESS\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/sql/ages_newlines.csv\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/userlib-0.1.zip\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/hello/\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/hello/sub_hello/\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/hello/sub_hello/sub_hello.txt\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/hello/hello.txt\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/test_pytorch_training_file.py\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/SimpleHTTPServer.py\n",
            "spark-3.4.0-bin-hadoop3/python/test_support/userlibrary.py\n",
            "spark-3.4.0-bin-hadoop3/python/mypy.ini\n",
            "spark-3.4.0-bin-hadoop3/python/docs/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/Makefile\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/_static/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/_static/css/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/_static/css/pyspark.css\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/_static/copybutton.js\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.ss/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.ss/query_management.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.ss/io.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.ss/core_classes.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.ss/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.mllib.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.ml.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/groupby.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/resampling.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/extensions.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/series.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/window.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/io.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/indexing.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/general_functions.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/frame.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.pandas/ml.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/spark_session.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/protobuf.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/row.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/avro.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/configuration.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/window.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/grouping.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/observation.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/functions.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/udf.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/io.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/column.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/catalog.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/core_classes.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/data_types.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.sql/dataframe.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.streaming.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.resource.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/reference/pyspark.errors.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/getting_started/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/getting_started/quickstart_connect.ipynb\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/getting_started/install.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/getting_started/quickstart_df.ipynb\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/getting_started/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/getting_started/quickstart_ps.ipynb\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/sql/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/sql/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/sql/arrow_pandas.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/best_practices.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/from_to_dbms.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/faq.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/typehints.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/options.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/types.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/transform_apply.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/pandas_pyspark.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/python_packaging.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/user_guide/arrow_pandas.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/_templates/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/_templates/autosummary/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/_templates/autosummary/class_with_docs.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/_templates/autosummary/class.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/conf.py\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/migration_guide/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/migration_guide/koalas_to_pyspark.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/migration_guide/pyspark_upgrade.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/migration_guide/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/development/\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/development/debugging.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/development/contributing.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/development/setting_ide.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/development/index.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/source/development/testing.rst\n",
            "spark-3.4.0-bin-hadoop3/python/docs/make.bat\n",
            "spark-3.4.0-bin-hadoop3/python/docs/make2.bat\n",
            "spark-3.4.0-bin-hadoop3/python/setup.py\n",
            "spark-3.4.0-bin-hadoop3/python/run-tests-with-coverage\n",
            "spark-3.4.0-bin-hadoop3/python/test_coverage/\n",
            "spark-3.4.0-bin-hadoop3/python/test_coverage/conf/\n",
            "spark-3.4.0-bin-hadoop3/python/test_coverage/conf/spark-defaults.conf\n",
            "spark-3.4.0-bin-hadoop3/python/test_coverage/sitecustomize.py\n",
            "spark-3.4.0-bin-hadoop3/python/test_coverage/coverage_daemon.py\n",
            "spark-3.4.0-bin-hadoop3/python/dist/\n",
            "spark-3.4.0-bin-hadoop3/python/README.md\n",
            "spark-3.4.0-bin-hadoop3/licenses/\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-istack-commons-runtime.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-machinist.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-AnchorJS.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-jaxb-runtime.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-arpack.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-pyrolite.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-leveldbjni.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-zstd.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-jsp-api.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-modernizr.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-jakarta.activation-api.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-jakarta.xml.bind-api.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-vis-timeline.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-xmlenc.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-reflectasm.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-pmml-model.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-jodd.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-bootstrap.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-mustache.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-slf4j.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-JTransforms.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-javassist.html\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-f2j.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-paranamer.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-d3.min.js.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-jakarta-ws-rs-api\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-matchMedia-polyfill.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-join.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-jline.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-blas.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-minlog.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-zstd-jni.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-spire.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-janino.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-sorttable.js.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-jquery.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-cloudpickle.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-kryo.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-javolution.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-datatables.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-graphlib-dot.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-dagre-d3.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-re2j.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-JLargeArrays.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-jakarta-annotation-api\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-sbt-launch-lib.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-antlr.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-protobuf.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-json-formatter.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-dnsjava.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-javax-transaction-transaction-api.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-CC0.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-respond.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-scopt.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-py4j.txt\n",
            "spark-3.4.0-bin-hadoop3/licenses/LICENSE-automaton.txt\n",
            "spark-3.4.0-bin-hadoop3/RELEASE\n",
            "spark-3.4.0-bin-hadoop3/sbin/\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-history-server.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-thriftserver.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/spark-daemons.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-thriftserver.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-mesos-dispatcher.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/decommission-slave.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-workers.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-mesos-shuffle-service.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/decommission-worker.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-slave.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-slave.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-workers.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-mesos-shuffle-service.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-connect-server.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-master.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/slaves.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-mesos-dispatcher.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-worker.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-slaves.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/workers.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-all.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/spark-config.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-master.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/spark-daemon.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-worker.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-all.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-slaves.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/start-history-server.sh\n",
            "spark-3.4.0-bin-hadoop3/sbin/stop-connect-server.sh\n",
            "spark-3.4.0-bin-hadoop3/README.md\n",
            "spark-3.4.0-bin-hadoop3/bin/\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-sql2.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/sparkR\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-class2.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/pyspark\n",
            "spark-3.4.0-bin-hadoop3/bin/run-example\n",
            "spark-3.4.0-bin-hadoop3/bin/sparkR2.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/docker-image-tool.sh\n",
            "spark-3.4.0-bin-hadoop3/bin/pyspark.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-class\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-submit\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-sql\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-shell\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-sql.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/find-spark-home\n",
            "spark-3.4.0-bin-hadoop3/bin/run-example.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/find-spark-home.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/load-spark-env.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/beeline.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-class.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-shell.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/load-spark-env.sh\n",
            "spark-3.4.0-bin-hadoop3/bin/beeline\n",
            "spark-3.4.0-bin-hadoop3/bin/sparkR.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-connect-shell\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-shell2.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/pyspark2.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-submit.cmd\n",
            "spark-3.4.0-bin-hadoop3/bin/spark-submit2.cmd\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-FU66EXfd1Rl"
      },
      "source": [
        "!pip install -q findspark"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jB-CCdMzd5GZ"
      },
      "source": [
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.4.0-bin-hadoop3\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OERndGb_d8zg"
      },
      "source": [
        "import findspark\n",
        "findspark.init()\n",
        "from pyspark.sql import SparkSession\n",
        "\n",
        "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = spark.read.csv('iris.CSV', inferSchema= True, header= True)\n",
        "df.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vCf4f4F-AkJ-",
        "outputId": "cd389daf-b7cc-4169-bf4c-54e8e6270c90"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+-------+-----------+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|variety|variety_num|\n",
            "+------------+-----------+------------+-----------+-------+-----------+\n",
            "|         5.1|        3.5|         1.4|        0.2| Setosa|          0|\n",
            "|         4.9|        3.0|         1.4|        0.2| Setosa|          0|\n",
            "|         4.7|        3.2|         1.3|        0.2| Setosa|          0|\n",
            "|         4.6|        3.1|         1.5|        0.2| Setosa|          0|\n",
            "|         5.0|        3.6|         1.4|        0.2| Setosa|          0|\n",
            "|         5.4|        3.9|         1.7|        0.4| Setosa|          0|\n",
            "|         4.6|        3.4|         1.4|        0.3| Setosa|          0|\n",
            "|         5.0|        3.4|         1.5|        0.2| Setosa|          0|\n",
            "|         4.4|        2.9|         1.4|        0.2| Setosa|          0|\n",
            "|         4.9|        3.1|         1.5|        0.1| Setosa|          0|\n",
            "|         5.4|        3.7|         1.5|        0.2| Setosa|          0|\n",
            "|         4.8|        3.4|         1.6|        0.2| Setosa|          0|\n",
            "|         4.8|        3.0|         1.4|        0.1| Setosa|          0|\n",
            "|         4.3|        3.0|         1.1|        0.1| Setosa|          0|\n",
            "|         5.8|        4.0|         1.2|        0.2| Setosa|          0|\n",
            "|         5.7|        4.4|         1.5|        0.4| Setosa|          0|\n",
            "|         5.4|        3.9|         1.3|        0.4| Setosa|          0|\n",
            "|         5.1|        3.5|         1.4|        0.3| Setosa|          0|\n",
            "|         5.7|        3.8|         1.7|        0.3| Setosa|          0|\n",
            "|         5.1|        3.8|         1.5|        0.3| Setosa|          0|\n",
            "+------------+-----------+------------+-----------+-------+-----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BegZo0dTf2tq",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e4576ae7-d93b-4483-adbf-f63f17c721e8"
      },
      "source": [
        "df.describe().show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+------------------+-------------------+------------------+------------------+---------+------------------+\n",
            "|summary|      sepal_length|        sepal_width|      petal_length|       petal_width|  variety|       variety_num|\n",
            "+-------+------------------+-------------------+------------------+------------------+---------+------------------+\n",
            "|  count|               150|                150|               150|               150|      150|               150|\n",
            "|   mean| 5.843333333333335|  3.057333333333334|3.7580000000000027| 1.199333333333334|     null|               1.0|\n",
            "| stddev|0.8280661279778637|0.43586628493669793|1.7652982332594662|0.7622376689603467|     null|0.8192319205190406|\n",
            "|    min|               4.3|                2.0|               1.0|               0.1|   Setosa|                 0|\n",
            "|    max|               7.9|                4.4|               6.9|               2.5|Virginica|                 2|\n",
            "+-------+------------------+-------------------+------------------+------------------+---------+------------------+\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NbgVsy6zB28S"
      },
      "source": [
        "## Практика 5. Дальше нужно собрать все признаки в единый вектор для работы машинного обучения.\n",
        "Для этого используется VectorAssembler\n",
        "https://spark.apache.org/docs/latest/ml-features.html#vectorassembler"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m-8QnsaSI0ub"
      },
      "source": [
        "from pyspark.ml.linalg import Vectors\n",
        "from pyspark.ml.feature import VectorAssembler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VJcLCkkHXBZD",
        "outputId": "3d2c5e52-5689-4440-c3b7-eed14d627f63"
      },
      "source": [
        "df.columns"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['sepal_length',\n",
              " 'sepal_width',\n",
              " 'petal_length',\n",
              " 'petal_width',\n",
              " 'variety',\n",
              " 'variety_num']"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W7VpjK2MWqJB"
      },
      "source": [
        "assembler = VectorAssembler(inputCols=['sepal_length', 'sepal_width','petal_length', 'petal_width'], outputCol='Vector')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RcVLMGYSWqW_"
      },
      "source": [
        "df_features = assembler.transform(df)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EFpTG9MwWqbe",
        "outputId": "364d3c5a-f214-4413-c503-b851c19c916d"
      },
      "source": [
        "df_features.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+-------+-----------+-----------------+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|variety|variety_num|           Vector|\n",
            "+------------+-----------+------------+-----------+-------+-----------+-----------------+\n",
            "|         5.1|        3.5|         1.4|        0.2| Setosa|          0|[5.1,3.5,1.4,0.2]|\n",
            "|         4.9|        3.0|         1.4|        0.2| Setosa|          0|[4.9,3.0,1.4,0.2]|\n",
            "|         4.7|        3.2|         1.3|        0.2| Setosa|          0|[4.7,3.2,1.3,0.2]|\n",
            "|         4.6|        3.1|         1.5|        0.2| Setosa|          0|[4.6,3.1,1.5,0.2]|\n",
            "|         5.0|        3.6|         1.4|        0.2| Setosa|          0|[5.0,3.6,1.4,0.2]|\n",
            "|         5.4|        3.9|         1.7|        0.4| Setosa|          0|[5.4,3.9,1.7,0.4]|\n",
            "|         4.6|        3.4|         1.4|        0.3| Setosa|          0|[4.6,3.4,1.4,0.3]|\n",
            "|         5.0|        3.4|         1.5|        0.2| Setosa|          0|[5.0,3.4,1.5,0.2]|\n",
            "|         4.4|        2.9|         1.4|        0.2| Setosa|          0|[4.4,2.9,1.4,0.2]|\n",
            "|         4.9|        3.1|         1.5|        0.1| Setosa|          0|[4.9,3.1,1.5,0.1]|\n",
            "|         5.4|        3.7|         1.5|        0.2| Setosa|          0|[5.4,3.7,1.5,0.2]|\n",
            "|         4.8|        3.4|         1.6|        0.2| Setosa|          0|[4.8,3.4,1.6,0.2]|\n",
            "|         4.8|        3.0|         1.4|        0.1| Setosa|          0|[4.8,3.0,1.4,0.1]|\n",
            "|         4.3|        3.0|         1.1|        0.1| Setosa|          0|[4.3,3.0,1.1,0.1]|\n",
            "|         5.8|        4.0|         1.2|        0.2| Setosa|          0|[5.8,4.0,1.2,0.2]|\n",
            "|         5.7|        4.4|         1.5|        0.4| Setosa|          0|[5.7,4.4,1.5,0.4]|\n",
            "|         5.4|        3.9|         1.3|        0.4| Setosa|          0|[5.4,3.9,1.3,0.4]|\n",
            "|         5.1|        3.5|         1.4|        0.3| Setosa|          0|[5.1,3.5,1.4,0.3]|\n",
            "|         5.7|        3.8|         1.7|        0.3| Setosa|          0|[5.7,3.8,1.7,0.3]|\n",
            "|         5.1|        3.8|         1.5|        0.3| Setosa|          0|[5.1,3.8,1.5,0.3]|\n",
            "+------------+-----------+------------+-----------+-------+-----------+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GjMxxrInEerA"
      },
      "source": [
        "## Разобьем данные на данные для обучения и проверки"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hj9FZxr-EXi0"
      },
      "source": [
        "train, test = df_features.randomSplit([0.8, 0.2], seed=12345)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rq_EQCXVFRJ3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d8b4ae34-889d-4c57-99ce-f0a926a31e83"
      },
      "source": [
        "train.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-----------+------------+-----------+----------+-----------+-----------------+\n",
            "|sepal_length|sepal_width|petal_length|petal_width|   variety|variety_num|           Vector|\n",
            "+------------+-----------+------------+-----------+----------+-----------+-----------------+\n",
            "|         4.3|        3.0|         1.1|        0.1|    Setosa|          0|[4.3,3.0,1.1,0.1]|\n",
            "|         4.4|        2.9|         1.4|        0.2|    Setosa|          0|[4.4,2.9,1.4,0.2]|\n",
            "|         4.4|        3.0|         1.3|        0.2|    Setosa|          0|[4.4,3.0,1.3,0.2]|\n",
            "|         4.4|        3.2|         1.3|        0.2|    Setosa|          0|[4.4,3.2,1.3,0.2]|\n",
            "|         4.5|        2.3|         1.3|        0.3|    Setosa|          0|[4.5,2.3,1.3,0.3]|\n",
            "|         4.6|        3.1|         1.5|        0.2|    Setosa|          0|[4.6,3.1,1.5,0.2]|\n",
            "|         4.6|        3.4|         1.4|        0.3|    Setosa|          0|[4.6,3.4,1.4,0.3]|\n",
            "|         4.6|        3.6|         1.0|        0.2|    Setosa|          0|[4.6,3.6,1.0,0.2]|\n",
            "|         4.7|        3.2|         1.3|        0.2|    Setosa|          0|[4.7,3.2,1.3,0.2]|\n",
            "|         4.7|        3.2|         1.6|        0.2|    Setosa|          0|[4.7,3.2,1.6,0.2]|\n",
            "|         4.8|        3.0|         1.4|        0.1|    Setosa|          0|[4.8,3.0,1.4,0.1]|\n",
            "|         4.8|        3.0|         1.4|        0.3|    Setosa|          0|[4.8,3.0,1.4,0.3]|\n",
            "|         4.8|        3.1|         1.6|        0.2|    Setosa|          0|[4.8,3.1,1.6,0.2]|\n",
            "|         4.8|        3.4|         1.6|        0.2|    Setosa|          0|[4.8,3.4,1.6,0.2]|\n",
            "|         4.8|        3.4|         1.9|        0.2|    Setosa|          0|[4.8,3.4,1.9,0.2]|\n",
            "|         4.9|        2.4|         3.3|        1.0|Versicolor|          1|[4.9,2.4,3.3,1.0]|\n",
            "|         4.9|        2.5|         4.5|        1.7| Virginica|          2|[4.9,2.5,4.5,1.7]|\n",
            "|         4.9|        3.0|         1.4|        0.2|    Setosa|          0|[4.9,3.0,1.4,0.2]|\n",
            "|         4.9|        3.1|         1.5|        0.1|    Setosa|          0|[4.9,3.1,1.5,0.1]|\n",
            "|         4.9|        3.1|         1.5|        0.2|    Setosa|          0|[4.9,3.1,1.5,0.2]|\n",
            "+------------+-----------+------------+-----------+----------+-----------+-----------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qjiImzktE3Rx"
      },
      "source": [
        "## Создадим и обучим модель логистической регрессии"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4QIgYk9VE1Ug"
      },
      "source": [
        "from pyspark.ml.classification import LogisticRegression"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DBcOuTtFFApc"
      },
      "source": [
        "lr = LogisticRegression(featuresCol = 'Vector', labelCol = 'variety_num')\n",
        "lrModel = lr.fit(train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tr9NN0dpFNTq"
      },
      "source": [
        "train_res = lrModel.transform(train)\n",
        "test_res = lrModel.transform(test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WlJPPs4rFrwv"
      },
      "source": [
        "## Оценим качество\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HlxLMOYAFpNt"
      },
      "source": [
        "from pyspark.ml.evaluation import MulticlassClassificationEvaluator"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6kDkBr1jGtv6"
      },
      "source": [
        "ev = MulticlassClassificationEvaluator(labelCol='variety_num')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yf1m2YR3G1oC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "446de5ef-bf55-4dc9-e533-34feff83cfcb"
      },
      "source": [
        "ev.evaluate(train_res)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9844961240310077"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OYffHnj9G_-m",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3c26f820-21d1-41e6-e404-ae9f4cbe7a3d"
      },
      "source": [
        "ev.evaluate(test_res)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IWiuZZ5SJUip"
      },
      "source": [
        "##Создадим модель дерева решений"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zHZB8WglLvh8"
      },
      "source": [
        "from pyspark.ml.classification import DecisionTreeClassifier"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EYObXRFXaFmc"
      },
      "source": [
        "tr = DecisionTreeClassifier(featuresCol = 'Vector', labelCol = 'variety_num')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DEeSdS9RaFzN"
      },
      "source": [
        "trFitted = tr.fit(train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rtSWJVynahNa"
      },
      "source": [
        "train_tr_res=trFitted.transform(train)\n",
        "test_tr_res=trFitted.transform(test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g5og8djwahUB",
        "outputId": "22e69e28-66b0-423e-f135-82213bc09a36"
      },
      "source": [
        "ev.evaluate(train_tr_res)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9922428036123125"
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CJHcr5Lwa-QS",
        "outputId": "6a089b3c-6f28-4306-e78f-69d0ec5e6f56"
      },
      "source": [
        "ev.evaluate(test_tr_res)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AlK_Jtj1J60h"
      },
      "source": [
        "## Вывод\n",
        "Обе модели хорошо подходят для решения данной задачи"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nrZ5gN71J41T"
      },
      "source": [],
      "execution_count": null,
      "outputs": []
    }
  ]
}